import urllib.request as ur
import lxml.etree as le
import re
import os
# https://so.csdn.net/so/search/s.do?p={page}&q={keyword}&t=blog&viparticle=&domain=&o=&s=&u=&l=&f=&rbg=0

#代理
def getaddress():
    proxy_address = ur.urlopen(
        'http://api.ip.data5u.com/dynamic/get.html?order=c1411144dca927c756f66be0c1e0ec81&sep=4').read().decode(
        'utf-8').strip()
    # 创建
    proxy_handler = ur.ProxyHandler(
        {
            'http': proxy_address
        }
    )

    proxy_opener = ur.build_opener(proxy_handler)

    return proxy_opener

url = 'https://so.csdn.net/so/search/s.do?p={page}&q={keyword}&t=blog&viparticle=&domain=&o=&s=&u=&l=&f=&rbg=0'


def getResponse(url):
    requset = ur.Request(
        url=url,
        headers={
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36'
        }
    )

    return requset


if __name__ == '__main__':
    keyword = input('关键词')
    pn_end = int(input('终止页'))

    for page in range(1, pn_end + 1):
        requset = getResponse(
            url='https://so.csdn.net/so/search/s.do?p={page}&q={keyword}&t=blog&viparticle=&domain=&o=&s=&u=&l=&f=&rbg=0'.format(
                page=page, keyword=keyword)
        )

        response = getaddress().open(requset).read()

        #print(response)

        hrefs = le.HTML(response).xpath('//div[@class="search-list-con"]/dl//span[@class="mr16"]/../../dt/div/a[1]/@href')

        for href in hrefs:

            response_blog = getaddress().open(getResponse(href)).read()

            #title = le.HTML(response_blog).xapth('//h1[@class="title-article"]/text()')[0]
            title = le.HTML(response_blog).xpath('//h1[@class="title-article"]/text()')[0]
            #a= ' / \ : * " < > | ?'

            title = re.sub(
                r'[/\\:*"<>|?]','',title
            )
            if not os.path.exists('csdn_blog'):
                os.mkdir('csdn_blog')

            filepath = 'csdn_blog/%s.html' % title
            with open(filepath,'wb') as f:
                f.write(response_blog)
                print(title)



